#!/usr/bin/python
#coding=utf-8

import sys
import os
import rados
import threading
import commands
import time
import copy
import logging
import logging.config

import global_list
from command import *

class ReqScrub:
    #self.h=链接集群的handle
    def __init__(self,handle,dict,reqid,logger):
        self.h=handle
        self.act=dict['act']
        self.item=dict['item']
        self.page=dict['page']
        self.pagesize=dict['pagesize']
        self.reqid=int(reqid)
        self.logger=logger

    def ThreadDealer(self):
        resp={}
        resp['act']=str(self.act)
        resp['type']='consistent'
        resp['status']='running'
        if self.act == 'notdone':
            resp['page']=self.page
        if self.act != 'getstats':
            resp['item']=self.item

        #tname=threading.currentThread().getName()
        mtx=global_list.get_value('MUTEX')
        #加锁->获取结果列表->追加结果->更新列表->解锁
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

        #执行具体命令
        if self.act == 'getstats':
            ret=self.getstats()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'enable':
            ret=self.enable()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'disable':
            ret=self.disable()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'notdone':
            ret=self.notdone()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'dodeepscrub':
            ret=self.dodeepscrub()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'doscrub':
            ret=self.doscrub()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        else:
            pass
 
        #更新结果
        mtx=global_list.get_value('MUTEX')
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

    #辅助函数，获取某个参数的value
    def get_value(self,host,key):
        retval=dict()
        cmd="/usr/bin/ssh "+str(host)+" /usr/bin/ceph daemon /var/run/ceph/ceph-mon."+str(host)+".asok config get "+str(key)+" -f json"
        ret,res=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" get config by asok failed, ret is "+str(ret))
            retval['err']='failed'
            return retval
        
        retval.update(json.loads(res))
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return retval
  
    #获取集群scrub和deep-scrub状态
    def getstats(self):
        respdict=dict()
        resplist=[]

        #查找一个mon主机地址
        mon_find = CephClusterCommand(self.h, prefix='mon_status', format='json')
        if mon_find.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" get mon_status failed, error is "+mon_find['err'])
            respdict['err']=mon_find['err']
            return respdict
        result=mon_find['result']
        host=result.get('monmap', 'None').get('mons', 'None')[0].get('name','None')
        self.logger.info("reqid:"+str(self.reqid)+" mon host is: "+str(host))

        #获取osd_scrub_begin_hour
        ret=self.get_value(host,'osd_scrub_begin_hour')
        if ret.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" get config osd_scrub_begin_hour failed")
            respdict['err']=ret['err']
            return respdict
        begin=ret['osd_scrub_begin_hour']

        #获取osd_scrub_end_hour
        ret=self.get_value(host,'osd_scrub_end_hour')
        if ret.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" get config osd_scrub_end_hour failed")
            respdict['err']=ret['err']
            return respdict
        end=ret['osd_scrub_end_hour']

        #获取osd_deep_scrub_interval
        ret=self.get_value(host,'osd_deep_scrub_interval')
        if ret.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" get config osd_deep_scrub_interval failed")
            respdict['err']=ret['err']
            return respdict
        interval=ret['osd_deep_scrub_interval']

        d1={'name':'scrub','status':'enabled'}
        d2={'name':'deepscrub','status':'enabled'}
        d1['osd_scrub_begin_hour']=d2['osd_scrub_begin_hour']=begin
        d1['osd_scrub_end_hour']=d2['osd_scrub_end_hour']=end
        
        #获取scrub/deep-scrub状态
        health_detail = CephClusterCommand(self.h, prefix='health', strings='detail', format='json')
        if health_detail.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" health detail failed, error is "+health_detail['err'])
            respdict['err']=health_detail['err']
            return respdict
        
        #分析summary数据
        result=health_detail['result']
        sumlist=result.get('summary')
        for i in sumlist:
            if -1 != i.get('summary','None').find('noscrub'):
                self.logger.info("reqid:"+str(self.reqid)+ " noscrub flag set")
                d1['status']='disabled'
            if -1 != i.get('summary','None').find('nodeep-scrub'):
                self.logger.info("reqid:"+str(self.reqid)+ " nodeep-scrub flag set")
                d2['status']='disabled'

        resplist.append(d1)
        resplist.append(d2)
        respdict['stats']=resplist
       
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict

    #开启某个状态
    def enable(self):
        if self.item == 'scrub':
           ret=os.system('/usr/bin/ceph osd unset noscrub')
           if ret == 0:
               self.logger.info("reqid:"+str(self.reqid)+" unset noscrub success")
               return True
           else:
               self.logger.error("reqid:"+str(self.reqid)+" unset noscrub failed")
               return False
        if self.item == 'deepscrub':
           ret=os.system('/usr/bin/ceph osd unset nodeep-scrub')
           if ret == 0:
               self.logger.info("reqid:"+str(self.reqid)+" unset nodeep-scrub failed")
               return True
           else:
               self.logger.error("reqid:"+str(self.reqid)+" unset nodeep-scrub failed")
               return False

    #关闭某个状态
    def disable(self):
        if self.item == 'scrub':
           ret=os.system('/usr/bin/ceph osd set noscrub')
           if ret == 0:
               self.logger.info("reqid:"+str(self.reqid)+" set noscrub success")
               return True
           else:
               self.logger.error("reqid:"+str(self.reqid)+" set noscrub failed")
               return False
        if self.item == 'deepscrub':
           ret=os.system('/usr/bin/ceph osd set nodeep-scrub')
           if ret == 0:
               self.logger.info("reqid:"+str(self.reqid)+" unset nodeep-scrub success")
               return True
           else:
               self.logger.error("reqid:"+str(self.reqid)+" unset nodeep-scrub failed")
               return False

    #辅助函数，获取第一个mon的host
    def get_mon_host(self):
        d=dict()

        mon_status = CephClusterCommand(self.h, prefix='mon_status', format='json')
        if mon_status.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" mon_status failed, error is "+mon_status['err'])
            d['err']=mon_status['err']
            return d

        result=mon_status['result']
        quorum=result.get('quorum')
        mons=result.get('monmap').get('mons')
        for i in mons:
            d['rank']=i.get('rank',-1)
            if d['rank'] in quorum:
                d['host']=i.get('name','None')
                break
            else:
                continue
       
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return d

    #查询集群内超期未进行scrub/deep-scrub的PGs
    def notdone(self):
        respdict=dict()
 
        #获取一个mon地址
        host=self.get_mon_host()
        if host.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" get mon host failed")
            respdict['err']=host['err']
            return respdict

        #获取osd_scrub_max_interval
        cmd="/usr/bin/ssh "+str(host['host'])+" /usr/bin/ceph daemon /var/run/ceph/ceph-mon."+str(host['host'])+".asok config get osd_scrub_max_interval -f json"
        ret,res=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" get config osd_scrub_max_interval by asok failed, ret is "+str(ret))
            respdict['err']='getceph daemon config osd_scrub_max_interval failed'
            return respdict
        d=json.loads(res)
        osd_scrub_max_interval=d['osd_scrub_max_interval']
        self.logger.info("reqid:"+str(self.reqid)+" osd_scrub_max_interval: "+str(osd_scrub_max_interval))

        #获取osd_deep_scrub_interval
        cmd="/usr/bin/ssh "+str(host['host'])+" /usr/bin/ceph daemon /var/run/ceph/ceph-mon."+str(host['host'])+".asok config get osd_deep_scrub_interval -f json"
        ret,res=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" get config osd_deep_scrub_interval by asok failed, ret is "+str(ret))
            respdict['err']='getceph daemon config osd_deep_scrub_interval failed'
            return respdict
        d=json.loads(res)
        osd_deep_scrub_interval=d['osd_deep_scrub_interval']
        self.logger.info("reqid:"+str(self.reqid)+" osd_deep_scrub_interval: "+str(osd_deep_scrub_interval))

        #获取pg dump pgs
        pg_dump = CephClusterCommand(self.h, prefix='pg dump', strings='pgs', format='json')
        if pg_dump.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" pg dump failed, error is "+pg_dump['err'])
            respdict['err']=pg_dump['err']
            return respdict
        pgs=pg_dump['result'].get('pg_stats','None')

        #将各个PGs的最后一次时间戳与当前时间进行对比
        now=time.time()
        if self.item == 'scrub':
            dumplist=[ x['pgid'] for x in pgs if float(now)-float(time.mktime(time.strptime(str(x['last_scrub_stamp']),"%Y-%m-%d %H:%M:%S.%f")))>float(osd_scrub_max_interval) ]
        else:
            dumplist=[ x['pgid'] for x in pgs if float(now)-float(time.mktime(time.strptime(str(x['last_deep_scrub_stamp']),"%Y-%m-%d %H:%M:%S.%f")))>float(osd_deep_scrub_interval) ]        
        start_idx=int(self.pagesize)*(int(self.page)-1)
        end_idx=start_idx+int(self.pagesize)
        respdict['count']=len(dumplist)
        respdict['lists']=dumplist[start_idx:end_idx]

        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict

    #手动对PG进行deep-scrub
    def dodeepscrub(self):
        retval=False
        difflist=[]

        #找到一个主机地址
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=0, format='json')
        if osd_find.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd find failed, error is "+osd_find['err'])
            return False
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
 
        #首先获取之前关联PGID的deep-scrub信息
        filename="/var/log/ceph/ceph.log"
        cmd="/usr/bin/ssh "+str(host)+" \'/usr/bin/grep -n \'"+str(self.item)+" deep-scrub\' "+filename+"\'"
        ret,before=commands.getstatusoutput(cmd)
        if 0 != ret:
            if before != '':
                self.logger.error("reqid:"+str(self.reqid)+" check ceph.log failed, ret is "+str(ret))
                return False
        
        #执行deep-scrub
        cmd='/usr/bin/ceph pg deep-scrub '+str(self.item)
        ret=os.system(cmd)
        if 0 != ret:
            self.logger.error("reqid:"+str(self.reqid)+" pg deep-scrub "+str(self.item)+" failed, ret is "+str(ret))
            return False

        beforelist=before.split('\n')
      
        #查询deep-scrub是否完成
        count=0
        cmd="/usr/bin/ssh "+str(host)+" \'/usr/bin/grep -n \'"+str(self.item)+" deep-scrub\' "+filename+"\'"
        while True:
            time.sleep(5)
            ret,now=commands.getstatusoutput(cmd)
            if 0 != ret:
                if now != '':
                    #此错误不退出，继续
                    self.logger.error("reqid:"+str(self.reqid)+" check ceph.log failed, ret is "+str(ret))
            nowlist=now.split('\n')
            for x in nowlist:
                if x not in beforelist:
                    self.logger.debug("reqid:"+str(self.reqid)+" found diff")
                    difflist.append(x)

            ok=str(self.item)+' deep-scrub ok'
            err=str(self.item)+' deep-scrub errors'
            for x in difflist:
                if x.find(ok) != -1:
                    self.logger.info("reqid:"+str(self.reqid)+" found ok key in diff")
                    retval=True
                    break
                elif x.find(err) != -1:
                    self.logger.info("reqid:"+str(self.reqid)+" found errors key in diff")
                    break
                else:
                    pass
            count+=1
            if count >= 100:
                self.logger.info("reqid:"+str(self.reqid)+" reach the top")
                break

        self.logger.info("reqid:"+str(self.reqid)+" ret="+str(retval))
        return retval

    #手动对PG进行scrub
    def doscrub(self):
        retval=False
        difflist=[]

        #找到一个主机地址
        osd_find = CephClusterCommand(self.h, prefix='osd find', id=0, format='json')
        if osd_find.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" osd find failed, error is "+osd_find['err'])
            return False
        result=osd_find['result']
        host=result.get('crush_location','None').get('host','None')
 
        #首先获取之前关联PGID的scrub信息
        filename="/var/log/ceph/ceph.log"
        cmd="/usr/bin/ssh "+str(host)+" \'/usr/bin/grep -n \""+str(self.item)+" scrub\" "+filename+"\'"
        ret,before=commands.getstatusoutput(cmd)
        if 0 != ret:
            if before != '':
                self.logger.error("reqid:"+str(self.reqid)+" check ceph.log failed, ret is "+str(ret))
                return False
        
        #执行scrub
        cmd='/usr/bin/ceph pg scrub '+str(self.item)
        ret=os.system(cmd)
        if 0 != ret:
            self.logger.error("reqid:"+str(self.reqid)+" pg scrub "+str(self.item)+" failed, ret is "+str(ret))
            return False

        beforelist=before.split('\n')
      
        #查询scrub是否完成
        count=0
        cmd="/usr/bin/ssh "+str(host)+" \'/usr/bin/grep -n \""+str(self.item)+" scrub\" "+filename+"\'"
        while True:
            time.sleep(5)
            ret,now=commands.getstatusoutput(cmd)
            if 0 != ret:
                if now != '':
                    #此错误不退出，继续
                    self.logger.error("reqid:"+str(self.reqid)+" check ceph.log failed, ret is "+str(ret))
            nowlist=now.split('\n')
            for x in nowlist:
                if x not in beforelist:
                    self.logger.debug("reqid:"+str(self.reqid)+" found diff")
                    difflist.append(x)

            ok=str(self.item)+' scrub ok'
            err=str(self.item)+' scrub errors'
            for x in difflist:
                if x.find(ok) != -1:
                    self.logger.info("reqid:"+str(self.reqid)+" found ok key in diff")
                    retval=True
                    break
                elif x.find(err) != -1:
                    self.logger.info("reqid:"+str(self.reqid)+" found errors key in diff")
                    break
                else:
                    pass
            count+=1
            if count >= 100:
                self.logger.info("reqid:"+str(self.reqid)+" reach the top")
                break

        self.logger.info("reqid:"+str(self.reqid)+" ret="+str(retval))
        return retval

    #析构
    def __del__(self):
        pass
